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APPENDIX A 

fs_purge.script 

isql -Usa -Pjc425 1 -e «! 

use faultdbl 

go 

drop proc fs_purgen 
go 

^** ************************** ******** 

** Create the PURGE stored procedure 

************************************/ 

CREATE procedure fs_purgen 
( 

@db_name varchar(30) ? 
@segment_name varchar(30), 
@space_left int, 
@status int 

) 

as 

/*** ***************************************************** *********** 

** purge procedure will now run every hour to avoid LONG 
** lockups at midnight. Just in case we have an extremely busy 
** hour, limit row count to 45k (15 traps/sec) to avoid running out 
** of locks 

*******************************************************************/ 

set rowcount 1500000 
DECLARE @activeAlarrnRetainDays int 
DECLARE @closedAlarmRetainDays int 
DECLARE @dateStamp varchar(40) 

SELECT @activeAlarrnRetainDays = (SELECT ActiveAlarmRetainDays 
FROM FaultServerConfig) 

SELECT @closedAlarmRetainDays - (SELECT ClosedAlarrnRetainDays 
FROM FaultServerConfig) 

SELECT @dateStamp - (getdate()) 

SET transaction isolation level 0 

PRINT "Purge started ... (%1 !)", @dateStamp 

PRINT "Purging old opentraps ... (%1 !)", @dateStamp 

BEGIN tran 

DECLARE @CustomerID binary(4) 
SELECT @CustomerID = 0 

DECLARE openPurgeAlarmKeyCur cursor FOR 

SELECT AlarmKey FROM OpenAlarm WHERE 

TimeStamp < dateadd(day, -@activeAlarrnRetainDays ? getdate()) 

OPEN openPurgeAlarmKeyCur 

DECLARE @temp AlarmKey numeric(30, 0) 
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FETCH openPurgeAlarmKeyCur into @tempAlarmKey 

WHILE (@@sqlstatus = 0) 

BEGIN 

UPDATE Alarm 
SET ActiveBit = 0 

WHERE AlarmKey = @tempAlarmKey 
DECLARE @RowKey_binbinary(4) 

SELECT @RowKey_bin = convert(binary(4),convert(int,@temp AlarmKey)) 
EXEC DBAction insert 'OpenAlanri, 'D, @RowKey_bin, @CustomerID 

FETCH openPurgeAlarmKeyCur into @tempAlarmKey 

END 



CLOSE OpenPurgeAlarmKeyCur 
DEALLOCATE cursor openPurgeAlarmKeyCur 

delete OpenTrap where TrapKey not in (select TrapKey from OpenEvent) 
COMMIT tran 



SET transaction isolation level 1 



** delete closed alarms that are older than the closedAlaimRetainDays and 
** the corresponding Events, GroupAlarms and Traps 



SELECT @dateStamp = (getdate()) 

PRINT "Purging old closed alarms ... (%1 !)", @dateStamp 

BEGIN tran 

DELETE Alarm from Alarm a WHERE 

TimeStamp < dateadd(day, -@closedAlanru^etairiDays ? getdate()) AND 
ActiveBit = 0 

COMMIT tran 



** delete all corresponding Events, GroupAlarms and Traps 



SELECT @dateStamp = (getdate()) 

PRINT "Purging old group alarms ... (%1 !)", @dateStamp 

BEGIN tran 

DELETE GroupAlarm WHERE 

GroupAlarmKey not in (SELECT AlarmKey FROM Alarm) 

COMMIT tran 



SELECT @dateStamp = (getdate()) 

PRINT "Purging old events ... (%1 !)", @dateStamp 

BEGIN tran 

DELETE Event FROM Trap t, Event e WHERE 
e.TrapKey = t.TrapKey AND 

t.TimeStamp < dateadd(day, -@closedAlarmRetainDays, getdate()) AND 
e.EventKey not in (SELECT EventKey FROM Alarm) AND 
e.EventKey not in (SELECT EventKey FROM GroupAlarm) 

COMMIT tran 

SELECT @dateStamp = (getdateQ) 
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PRINT "Purging old traps ... (%1 !)", @dateStamp 

BEGIN tran 

DELETE Trap WHERE TrapKey not in (SELECT TrapKey FROM Event) AND 
TimeStamp < dateadd(day, -@closedAlarmRetainDays, getdate()) 

COMMIT tran 

SELECT @dateStamp = (getdate()) 

PRINT "Purging old CircuitAlarms ... (%1 !)", @dateStamp 

DELETE CircuitAlarm WHERE datediff(day,TimeStamp,getdate()) 
> (select MaxCircuitAlarmDays from FaultServerConfig) 

/********************************************************************* 

** MaxCircuitAlarmDays is used for SLA reports 

** > @closedAlarrnRetainDays 

****************************************************** 

SELECT @dateStamp - (getdate()) 

PRINT "Purge completed ... (%1 !)", @dateStamp 

go 
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fs inserts. script 

isql -Usa -Pjc4251 -e «! 

use faultdbl 

go 

drop proc fs_inserts 
go 

** Create the STATS stored procedure 
CREATE procedure fs inserts 

( 

@db_name varchar(30), 
@segment_name varchar(30), 
@space_left int, 
@status int 

) 

as 

/** ***************************************** ******************** 

** count the number of traps and alarms inserted during the last 
** hour. 

DECLARE @alarmRows int 
DECLARE @trapRows int 
DECLARE @timeStamp varchar(40) 
DECLARE @dateStamp varchar(40) 

SELECT @timeStamp = (dateadd(hour, -1, getdate())) 
SELECT @dateStamp = (getdate()) 

SELECT @alarmRows = (SELECT count(*) from Alarm where 
TimeStamp > @timeStamp) 

SELECT @trapRows - (SELECT count(*) from Trap where 
TimeStamp > @timeStamp) 

PRINT "Insert %1! Alarms since %2! (%3!)", 

(gialarmRows, @timeStamp, @dateStamp 

PRINT "Insert %1 ! Traps since %2! (%3 !)", 

@trapRows, @timeStamp 3 @dateStamp 

go 
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APPENDIX C 

fs stats, script 

isql-Usa -Pjc4251 -e «! 

use faultdbl 

go 

drop proc fs stats 
go 

y'*** ***!&:******************** ********* 

** Create the STATS stored procedure 
************************************/ 

CREATE procedure fs_stats 
( 

@db_name varchar(30), 
@segment_name varchar(30), 
@space_left int, 
@status int 

) 

as 

/** ************************** *************************************** 

** purge procedure will now run every hour to avoid LONG 
** lockups at midnight. Just in case we have an extremely busy 
** hour, limit row count to 45k (15 traps/sec) to avoid running out 
** of locks 

*******************************************************************/ 

DECLARE @alarmRows int 
DECLARE @trapRows int 
DECLARE @timeStamp varchar(40) 
DECLARE @dateStamp varchar(40) 

SELECT @timeStamp = (dateadd(day, -1, getdate())) 
SELECT @dateStamp = (getdate()) 

SELECT @alarmRows = (SELECT count(*) from Alarm where 
Time Stamp < dateadd(day, -1, getdate())) 

SELECT @trapRows = (SELECT count(*) from Trap where 
TimeStamp < dateadd(day, -1, getdate())) 

PRINT "%1 ! Alarms older than %2! (%3 !)", 

@alarmRows, @timeStamp, @dateStamp 
PRINT Traps older than %2! (%3!)", 

@trapRows, @timeStamp ? @dateStamp 

SELECT @alarmRows = (SELECT count(*) from Alarm) 

SELECT @trapRows = (SELECT count(*) from Trap) 

PRINT "% 1 ! Total Alarms 
PRINT "%1 ! Total Traps 

go 



(%2!)", @alarmRows, @dateStamp 
(%2!)" 5 @trapRows, @dateStamp 
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APPENDIX D 

fs stats hr . script 

isql -Usa -Pjc4251 -e «! 

use faultdbl 

go 

drop proc fs_stats_hr 
go 

^** ************************** ******** 
** Create the STATS stored procedure 

CREATE procedure fs_stats_hr 
( 

@db_name varchar(30), 
@segment_name varchar(30), 
@space_left int, 
@status int 

) 

as 

/********************************** ********************************* 

** purge procedure will now run every hour to avoid LONG lockups at midnight. Just in case we have 

** extremely busy hour, limit row count to 45k (15 traps/sec) to avoid running out of locks 
*******************************************************************/ 

DECLARE @alarmRows int 
DECLARE @trapRows int 
DECLARE @timeStamp varchar(40) 
DECLARE @dateStamp varchar(40) 
DECLARE @activeAlarmRetainDays int 
DECLARE @closedAlarmRetainDays int 

SELECT @dateStamp - (getdate()) 

SELECT (gactiveAlarrnRetainDays = (SELECT ActiveAlarmRetainDays 
FROM FaultServerConfig) 

SELECT @closedAlarmRetainDays = (SELECT ClosedAlarmRetainDays 
FROM FaultServerConfig) 

SELECT @timeStamp = (dateadd(day, -@activeAlarmRetainDays 3 getdate())) 

PRINT ir activeAlarmRetainDays%l! n ; @activeAlarmRetainDays 
PRINT ''ClosedAlarmRetainDays %1!'\ ©ClosedAlarmRetainDays 

SELECT @alarmRows = (SELECT count(*) from Alarm where 

TimeStamp < dateadd(day, -@closedAlarmRetainDays, getdate())) 

SELECT @trapRows = (SELECT count(*) from Trap where 

TimeStamp < dateadd(day, -@closedAlarmRetainDays ? getdate())) 

PRINT "Delete %1 ! Alarms before %2! (%3 !)", 

@alarmRows 3 @timeStamp, @dateStamp 
PRINT "Delete %1 ! " Traps before %2! (%3!)", 

@trapRows, @timeStamp, @date Stamp 

go 
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APPENDIX E 

fault cron 

#!/bin/ksh 
Uname="uname -n 



reset_all () 

{ 



export cnt_TrapForw cnt RuleHand cnt Reliable cntDBIdHand cnt fstrapd 
export cnt Notifica cnt_AlarmHan cnt_ControlI cntAlarrnFor cntEventHan 

cnt_TrapForw=0 

cnt_RuleHand=0 

cnt_Reliable=0 

cnt_DBIdHand=0 

cnt_fstrapd=0 

cnt_Notifica=0 

cnt_AlarrnHan=0 

cnt_ControlI=0 

cntAlarrnFor^O 

cnt EventHan=0 



} 



startjt () 
{ 



cnt=0 
export cnt 

Date= date +'%b %e %T r 
/etc/rc3.d/S99jws stop 

echo "SDate SUname fault_cron: /etc/rc3.d/S99jws stop" \ 
»/var/adm/messages 

while true 
do 

if ps -fe | grep -i web | grep -v grep 
then 

cnt="expr $cnt + V 

if test $cnt-gt5 
then 

for x in 'ps -fe | grep -i web | \ 

grep -v grep | awk '{print $2} n 

do 

done 
break 

fi 

sleep 2 
continue 



kill $x 



fi 

break 

done 

/etc/rc3.d7S77fsd stop 
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sleep 2 

cd /opt/RelyENT/bin/ 
/opt/RelyENT/bin/StopAgt.sh 

echo "SDate $Uname fault_cron: /etc/rc3.d/S77fsd stop" \ 
»/var/adm/messages 

while true 
do 

if ps -fe | grep -i cvFault | grep -v grep 
then 

cnt='expr $cnt + V 

if test $cnt-gt 5 
then 

for x in 'ps -fe | grep -i cvFault | \ 

grep -v grep | awk '{print $2} p 

do 

Processes -e|grep " $x "|awk '{print $4}' v 

echo "SDate SUname fault cron: killing SProcess" \ 

»/var/ adm/messages 
kill -9 $x 

done 
break 

fi 

sleep 2 
continue 

fi 

break 

done 

/etc/rc3.d/S77fsd start 

echo "$Date SUname fault_cron: /etc/rc3.d/S77fsd start" \ 
»/var/adm/messages 

sleep 4 

/etc/rc3.d/S99jws start 
sleep 2 

/opt/RelyENT/bin/StartAgt.sh 

echo "SDate SUname fault_cron: /etc/rc3.d/S99jws start" \ 
»/var/adm/messages 

reset_all 
sleep 2 

} 

String-"RuleHan|DBIdHan|EventHa|AlarmHa|TrapForw|Reliabl|fe 
Date= date +'%b %e %T r 

echo "SDate SUname fault_cron: Starting" >>/var/adrn/messages 
reset_all 

while true 
do 

ps -e|egrep "SString" | awk '{print $4}' >/tmp/res.$$ 
Date='date +'%b %e %T" 
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##for x in TrapForw RuleHand Reliable DBIdHand fstrapd Notifica \ 
##for x in TrapForw RuleHand Reliable DBIdHand fstrapd Notifica \ 
##AlarmHan Controll AlarmFor EventHan 

for x in TrapForw RuleHand DBIdHand fstrapd Notifica \ 
AlarmHan AlarmFor EventHan 

do 

if grep $x /tmp/res.$$ >/dev/null 
then 

else 

case $x in 

TrapForw) cntJTrapForw^expr ScntTrapForw + V 

echo "SDate SUname fault__cron: $x stopped ScntJTrapForw" \ 
»/var/adm/messages ;; 
RuleHand) cnt_RuleHand= x expr ScntRuleHand + V 

echo "SDate SUname fault_cron: $x stopped $cnt_RuleHand" \ 
»/var/adm/messages 
Reliable) cnt_Reliable= expr $cnt_Reliable + V 

echo "SDate SUname fault_cron: $x stopped $cnt_Reliable" \ 
»/var/adm/messages 
DBIdHand) cnt_DBIdHand= expr ScntDBIdHand + V 

echo "SDate SUname fault cron: $x stopped Scnt DBIdHand" \ 
»/var/adm/messages 
fstrapd) cnt_fstrapd="expr Scnt_fstrapd + V ; 

echo "SDate SUname fault_cron: $x stopped $cnt_fstrapd" \ 
»/var/adm/messages 
Notifica) cnt_NotifIca= v expr $cnt_Notifica + V 

echo "SDate SUname fault_cron: $x stopped $cnt_Notifica" \ 
»/var/adm/messages ;; 
AlarmHan) cnt_AlarmHan='expr $cnt_AlarmHan + V 

echo "SDate SUname fault_cron: $x stopped $cnt_AlarmHan" \ 
»/var/adm/messages 
Controll) cnt ControlI^'expr $cnt_ControlI + V 

echo "SDate SUname fault_cron: $x stopped Scnt ControlI" \ 
»/var/adm/messages 
AlarmFor) cnt_AlarmFor='expr Scnt AlarmFor 4- V 

echo "SDate SUname fault_cron: $x stopped Scnt AlarmFor" \ 
»/var/adm/messages 
EventHan) cnt_EventHan= x expr Scnt EventHan + V 

echo "SDate SUname fault_cron: $x stopped $cnt_EventHan" \ 
»/var/adm/messages ;; 

esac 

fi 

done 

if test Scnt_TrapForw -ge 10 
then 

start_it 

elif test $cnt_RuleHand -ge 10 
then 

start_it 

elif test $cnt_Reliable -ge 10 
then 

start_it 

elif test ScntJDBIdHand -ge 10 
then 

start_it 
elif test Scnt_fstrapd -ge 10 
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then 

start_it 

elif test $cnt_Notifica -ge 10 
then 

start_it 

elif test $cnt_AlarmHan -ge 10 
then 

start_it 

elif test $cnt_ControlI -ge 10 
then 

start_it 

elif test $cnt_AlarmFor -ge 10 
then 

start_it 

elif test Scnt EventHan -ge 10 
then 

startjft 

else 

sleep 2 

fi 

done 

rm-f /tmp/res.$$ 
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APPENDIX F 

check inserts. sh 

Uname^uname -n' 
Prog="basename $0" 

PURGELOG=/opt/cvFaultServer/log/fsPurge.log 
CHECKLOG=/opt/cvFaultServer/log/check_insert.log 
STARS=" *************************" 

Date= date '+%b %e %Y %l:%p , |sed *s/ / /g r 

Alarmcnt=*grep '"Tnsert " SPURGELOG | tail -2 | grep Alarm | wc -F 
nbrAlarms= grep " A Insert " SPURGELOG | \ 
tail -2 | grep Alarm | awk '{print $2} n 

Alarmdate^ grep ,,A Insert " SPURGELOG | \ 

tail -2 | grep Alarm | awk '{print $9,$10,$1 1,$12}' | \ 
sed 's/[0]//g' I sed V:J'f 

Trapcnt=^grep '"Tnsert " SPURGELOG | tail -2 | grep Trap | wc -Y 
nbrTraps= grep " A Insert " SPURGELOG | \ 
tail -2 | grep Trap | awk '{print $2} r 

Trapdate= grep " A Insert " SPURGELOG | \ 

tail -2 | grep Trap | awk '{print $9,$10,$1 1,$12}* | \ 
sed's/fOy/gMsed 's/U'f 

if test SAlarmcnt != 1 -a STrapcnt != 1 
then 

echo "fs inserts not running" »$CHECKLOG 
exit 1 

fi 

if test "SAlarmdate" != "SDate" -a "STrapdate" != "SDate" 
then 

echo "date mismatch" »$CHECKLOG 
exit 1 

fi 

if test SnbrAlarms -eq 0 -a SnbrTraps -eq 0 
then 

echo "Mate +'%b %e %T r SUname SProg: Alarms and Traps have stopped" \ 

»$PURGELOG 
echo "SSTARSSSTARSSSTARS" »$PURGELOG 

echo "Mate +'%b %e %T r SUname SProg: Alarms and Traps have stopped" \ 

»$CHECKLOG 
echo "SSTARSSSTARSSSTARS" »$CHECKLOG 

/usr/local/bin/rmcore 

fault_cron= ps -fe | grep fault_cron | egrep -v "grep" | \ 
awk '{print S2} r 

iftestX"$fault_cron" !=X 
then 

echo "Mate +'%b %e %T r SUname SProg: kill fault_cron\t$fault_cron" \ 
»$CHECKLOG 
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kill -9 Sfault cron 



cnt=0 
while true 
do 



iftest$cnt-gt2 
then 

break 

fi 

fault_cron= ps -fe | grep fault_cron | egrep -v "grep" | \ 
awk '{print $2} r 

iftestX"$fault_cron" !=X 
then 

echo "Mate +'%b %e %T" $Uname $Prog: kill fault__cron\t$fault_cron" \ 

»$CHECKLOG 
kill -9 $fault_cron 
break 

fi 

cnr='expr Sent + V 
sleep 4 



done 



cnt=0 
export cnt 

Date= date +'%b %e %T 

echo "$Date $Uname $Prog: /etc/rc3.d/S99jws stop" »$CHECKLOG 
/etc/rc3.d/S99jws stop »$CHECKLOG 2>&1 



while true 
do 



if ps -fe | grep -i web | grep -v grep 
then 

cnt='expr Sent + V 

if test $cnt-gt 5 
then 

for x in 'ps -fe | grep -i web | \ 

grep -v grep | awk '{print $2} r 



do 



iftestX"$x" !=X 
then 

Process^ ps -e | grep " $x " | awk '{print $4}" 
echo "Mate +'%b %e %T n $Uname $Prog: kill $Process\t$x" \ 
»$CHECKLOG 
kill -9 $x 

fi 



done 
break 

fi 

sleep 5 
continue 

fi 

break 



done 
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echo "SDate SUname $Prog: /etc/rc3.d/S77fsd stop" \ 

»$CHECKLOG 
/etc/rc3.d/S77fsd stop »$CHECKLOG 2>&1 

while true 
do 

if ps -fe | grep -i cvFault | grep -v grep 
then 

cnt^expr $cnt + V 

if test Scnt-gt5 
then 

for x in v ps -fe | grep -i cvFault | \ 

egrep -v "fsPurge|check_insert|grep" | awk '{print $2} p 

do 

iftestX"$x" !=X 
then 

Process="ps -e [ grep " $x " | awk '{print $4} r 
echo "Mate +'%b %e %T n SUname SProg: kill $Process\t$x" \ 
»$CHECKLOG 
kill -9 $x 

fi 

done 
break 

fi 

sleep 5 
continue 

fi 

break 

done 

echo "SDate $Uname $Prog: /etc/rc3.d/S77fsd start" \ 

»$CHECKLOG 
/etc/rc3.d/S77fsd start »$CHECKLOG 2>&1 

sleep 4 

echo "SDate SUname SProg: /etc/rc3.d/S99jws start" \ 

»$CHECKLOG 
/etc/rc3.d/S99jws start »$CHECKLOG 2>&1 

echo "SDate SUname SProg: Starting fault_cron" \ 

»$CHECKLOG 
/usr/local/bin/fault_cron & 

echo "SDate SUname SProg: Restart complete" \ 

»$CHECKLOG 
echo "$STARS$STARS************" \ 

»SCHECKLOG 

echo "Okydoky" »$PURGELOG 
echo "Okydoky" »$CHECKLOG 
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APPENDIX G 

fspurge.sh 

#!/bin/sh 

# Filename: fsPurge. sh 

# Description: 

# script to invoke a Sybase store procedure to purge the faultdb. 

USAGE- Usage: fsPurge.sh server db_name username password sybasePath' 
Uname="uname -n 
Prog="basename $0' 

############################## 

# check command line arguments 

if [$#-ne5] 
then 

echo SUSAGE 
exit 1 

fi 

######### // // # ########################## 

# log all outputs from isql statements 

exec »/opt/cvFaultServer/log/fsPurge.log 2>&1 

SYBASE=$5 

export SYBASE 

DSQUERY=$1 

export DSQUERY 

BIN=/opt/cvFaultServer/bin 

LOCAL=/usr/local/bin 

LOG=/opt/cvFaultServer/log 

echo "\n date +'%b %e %T r $Uname $Prog: Started" 
echo M *********************************^ 

########^^ 

# check and make sure fsPurge, sh is not already running ! 

ps -e | grep fsPurge. 

cnt="ps -e | grep fsPurge, | wc -V 

if test $cnt-ge2 
then 

echo "Mate +*%b %e %T n $Uname $Prog: aheady ruiming" 
exit 1 

fi 

# The grep will remove output lines that contain a number surrounded by 

# any amount of whitespace. If the number is bounded by words, then it 

# will not be removed from the output stream. 

// J! if .'i if n a n i! a ii ii ii a a u n n n tr n tin u u a n n n it it n 11 it tin 11 tut n ujj u n n n n u n'! n u u n n n it i t it it it it n n 1 1 a i t n it a n 

II It 1 1 II If II II II It if ti ti TV it It ft It It It 11 II TTTtftfrrrtl 1 1 it ft If ft ft ft ft ft ft ft tf ft nil II Irtl !! trtTTTTTtTTT!' !/)!!/ tit) )! ft It II I) II Hit It It It II II 

Status=0; export Status 
Count=0 



23 



BELL-0117/01115 



PATENT 



m 



##############################################^ 

# test for successful conclusion, i.e. no deadlock, for up to 

# 10 times (10 minutes) ! 

$5/bin/isql -U$3 -P$4 2>&1 «! | egrep -v -f $BIN/exclude 

use $2 
go 

exec fs_statsjir $2, 0, 0 
go 



while true 
do 

echo "\n date +'%b %e %T SUname SProg:" 

$5/bin/isql -U$3 -P$4 2>&1 «! | egrep -v -f SBIN/exclude 

use $2 
go 

exec fs_purgen $2, "", 0, 0 
go 



Status= x tail /opt/cvFaultServer/log/f sPurge.log | \ 
grep " A (return status" | tail -T 

###################### 

# did purge succeed ?? 

if test X"$Status" - X"(rerurn status = 0)" 
then 

sleep 5 
break 

fi 

Counts expr SCount + V 

######################## 

# try up to 10 more times 

if test SCount -ge 10 
then 

echo "Purge failed at Mate +'%b %e %T M ' 

sleep 5 

break 

fi 



done 



sleep 60 



echo "\n date +'%b %e %T" SUname SProg" 
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$5/bin/isql -U$3 -P$4 «! | egrep -v -f $BIN/exclude 

spjielpdb $2 

go 
i 

echo "\n date +'%b %e %T r SUname SProg" 

$5/bin/isql -U$3 -P$4 «! ] egrep -v -f $BIN/exclude 

use $2 
go 

exec fsjnserts $2 5 0, 0 

go 
i 

nohup $LOCAL/check_inserts.sh »/opt/cvFaultServer/log/check_insert.log 2>&1 & 
echo "\n date +'%b %e %T r $Uname SProg: Completed" 

if test -f$LOG/DBId* 
then 

df-k /opt 

rm-f$LOG/*Buffer* 

df-k /opt 

echo 

fl 
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